{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab Size: 19124 | x_train: 211727 | x_test: 47377\n",
      "(211707, 20) (211707, 20) (2368, 20) (2368, 20)\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py:497: calling conv1d (from tensorflow.python.ops.nn_ops) with data_format=NHWC is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "`NHWC` for data_format is deprecated, use `NWC` instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:98: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Shuffled\n",
      "Epoch 1/1 | Step 0/1653 | train_loss: 75.9255 | train_acc: 0.0348 | lr: 0.0050\n",
      "Epoch 1/1 | Step 50/1653 | train_loss: 3.5961 | train_acc: 0.9508 | lr: 0.0047\n",
      "Epoch 1/1 | Step 100/1653 | train_loss: 1.2834 | train_acc: 0.9789 | lr: 0.0044\n",
      "Epoch 1/1 | Step 150/1653 | train_loss: 0.7434 | train_acc: 0.9898 | lr: 0.0041\n",
      "Epoch 1/1 | Step 200/1653 | train_loss: 0.6255 | train_acc: 0.9918 | lr: 0.0038\n",
      "Epoch 1/1 | Step 250/1653 | train_loss: 0.2492 | train_acc: 0.9957 | lr: 0.0035\n",
      "Epoch 1/1 | Step 300/1653 | train_loss: 0.4201 | train_acc: 0.9926 | lr: 0.0033\n",
      "Epoch 1/1 | Step 350/1653 | train_loss: 0.1966 | train_acc: 0.9965 | lr: 0.0031\n",
      "Epoch 1/1 | Step 400/1653 | train_loss: 0.1028 | train_acc: 0.9988 | lr: 0.0029\n",
      "Epoch 1/1 | Step 450/1653 | train_loss: 0.2348 | train_acc: 0.9961 | lr: 0.0027\n",
      "Epoch 1/1 | Step 500/1653 | train_loss: 0.1731 | train_acc: 0.9969 | lr: 0.0025\n",
      "Epoch 1/1 | Step 550/1653 | train_loss: 0.1469 | train_acc: 0.9969 | lr: 0.0023\n",
      "Epoch 1/1 | Step 600/1653 | train_loss: 0.1296 | train_acc: 0.9988 | lr: 0.0022\n",
      "Epoch 1/1 | Step 650/1653 | train_loss: 0.1556 | train_acc: 0.9977 | lr: 0.0020\n",
      "Epoch 1/1 | Step 700/1653 | train_loss: 0.0850 | train_acc: 0.9984 | lr: 0.0019\n",
      "Epoch 1/1 | Step 750/1653 | train_loss: 0.2323 | train_acc: 0.9957 | lr: 0.0018\n",
      "Epoch 1/1 | Step 800/1653 | train_loss: 0.1512 | train_acc: 0.9969 | lr: 0.0016\n",
      "Epoch 1/1 | Step 850/1653 | train_loss: 0.1636 | train_acc: 0.9973 | lr: 0.0015\n",
      "Epoch 1/1 | Step 900/1653 | train_loss: 0.0886 | train_acc: 0.9996 | lr: 0.0014\n",
      "Epoch 1/1 | Step 950/1653 | train_loss: 0.0782 | train_acc: 0.9988 | lr: 0.0013\n",
      "Epoch 1/1 | Step 1000/1653 | train_loss: 0.1736 | train_acc: 0.9969 | lr: 0.0012\n",
      "Epoch 1/1 | Step 1050/1653 | train_loss: 0.1298 | train_acc: 0.9973 | lr: 0.0012\n",
      "Epoch 1/1 | Step 1100/1653 | train_loss: 0.0608 | train_acc: 0.9992 | lr: 0.0011\n",
      "Epoch 1/1 | Step 1150/1653 | train_loss: 0.0907 | train_acc: 0.9980 | lr: 0.0010\n",
      "Epoch 1/1 | Step 1200/1653 | train_loss: 0.1309 | train_acc: 0.9973 | lr: 0.0009\n",
      "Epoch 1/1 | Step 1250/1653 | train_loss: 0.0669 | train_acc: 0.9988 | lr: 0.0009\n",
      "Epoch 1/1 | Step 1300/1653 | train_loss: 0.0726 | train_acc: 0.9984 | lr: 0.0008\n",
      "Epoch 1/1 | Step 1350/1653 | train_loss: 0.0623 | train_acc: 0.9984 | lr: 0.0008\n",
      "Epoch 1/1 | Step 1400/1653 | train_loss: 0.0587 | train_acc: 0.9984 | lr: 0.0007\n",
      "Epoch 1/1 | Step 1450/1653 | train_loss: 0.0800 | train_acc: 0.9988 | lr: 0.0007\n",
      "Epoch 1/1 | Step 1500/1653 | train_loss: 0.1327 | train_acc: 0.9980 | lr: 0.0006\n",
      "Epoch 1/1 | Step 1550/1653 | train_loss: 0.1327 | train_acc: 0.9973 | lr: 0.0006\n",
      "Epoch 1/1 | Step 1600/1653 | train_loss: 0.0528 | train_acc: 0.9992 | lr: 0.0005\n",
      "Epoch 1/1 | Step 1650/1653 | train_loss: 0.0936 | train_acc: 0.9977 | lr: 0.0005\n",
      "Epoch 1/1 | train_loss: 0.1016 | train_acc: 0.9984 | lr: 0.0005\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      <pad>       0.91      0.94      0.92      6639\n",
      "         NN       0.99      1.00      0.99      5070\n",
      "         IN       1.00      1.00      1.00      4020\n",
      "         DT       0.97      0.94      0.95       912\n",
      "        VBZ       0.97      0.94      0.96      1354\n",
      "         RB       0.90      0.89      0.90      1103\n",
      "        VBN       1.00      1.00      1.00      1177\n",
      "         TO       0.93      0.95      0.94      1269\n",
      "         VB       0.88      0.92      0.90      2962\n",
      "         JJ       0.97      0.90      0.93      3034\n",
      "        NNS       0.94      0.92      0.93      4803\n",
      "        NNP       1.00      1.00      1.00      2389\n",
      "          ,       1.00      1.00      1.00      1214\n",
      "         CC       1.00      1.00      1.00       433\n",
      "        POS       1.00      1.00      1.00      1974\n",
      "          .       0.90      0.92      0.91       539\n",
      "        VBP       0.91      0.88      0.90       727\n",
      "        VBG       1.00      1.00      1.00       421\n",
      "       PRP$       0.97      0.97      0.97      1918\n",
      "         CD       1.00      1.00      1.00       323\n",
      "         ``       1.00      1.00      1.00       316\n",
      "         ''       0.96      0.94      0.95      1679\n",
      "        VBD       1.00      1.00      1.00        48\n",
      "         EX       1.00      0.99      0.99       470\n",
      "         MD       1.00      1.00      1.00        11\n",
      "          #       1.00      1.00      1.00        77\n",
      "          (       1.00      1.00      1.00       384\n",
      "          $       1.00      1.00      1.00        77\n",
      "          )       0.89      0.62      0.73       130\n",
      "       NNPS       1.00      1.00      1.00       814\n",
      "        PRP       1.00      0.94      0.97        77\n",
      "        JJS       1.00      1.00      1.00       110\n",
      "         WP       0.94      0.87      0.90        70\n",
      "        RBR       0.94      0.96      0.95       202\n",
      "        JJR       0.97      0.94      0.96       202\n",
      "        WDT       1.00      0.99      0.99        93\n",
      "        WRB       1.00      0.98      0.99        49\n",
      "        RBS       1.00      0.90      0.95        10\n",
      "        PDT       1.00      0.50      0.67        12\n",
      "         RP       1.00      1.00      1.00       238\n",
      "          :       1.00      0.75      0.86         4\n",
      "         FW       1.00      1.00      1.00         4\n",
      "        WP$       1.00      0.50      0.67         2\n",
      "\n",
      "avg / total       0.96      0.96      0.96     47360\n",
      "\n",
      "I love you\n",
      "PRP VBP PRP\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1428: UserWarning: labels size, 43, does not match size of target_names, 45\n",
      "  .format(len(labels), len(target_names))\n"
     ]
    }
   ],
   "source": [
    "import pos\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from cnn_seq_label import Tagger\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "\n",
    "SEQ_LEN = 20\n",
    "BATCH_SIZE = 128\n",
    "NUM_EPOCH = 1\n",
    "sample = ['I', 'love', 'you']\n",
    "\n",
    "\n",
    "def to_train_seq(*args):\n",
    "    data = []\n",
    "    for x in args:\n",
    "        data.append(iter_seq(x))\n",
    "    return data\n",
    "\n",
    "\n",
    "def to_test_seq(*args):\n",
    "    data = []\n",
    "    for x in args:\n",
    "        x = x[: (len(x) - len(x) % SEQ_LEN)]\n",
    "        data.append(np.reshape(x, [-1, SEQ_LEN]))\n",
    "    return data\n",
    "\n",
    "\n",
    "def iter_seq(x, text_iter_step=1):\n",
    "    return np.array([x[i : i+SEQ_LEN] for i in range(0, len(x)-SEQ_LEN, text_iter_step)])\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    x_train, y_train, x_test, y_test, vocab_size, n_class, word2idx, tag2idx = pos.load_data()\n",
    "    X_train, Y_train = to_train_seq(x_train, y_train)\n",
    "    X_test, Y_test = to_test_seq(x_test, y_test)\n",
    "    print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape)\n",
    "\n",
    "    clf = Tagger(vocab_size, n_class, SEQ_LEN)\n",
    "    clf.fit(X_train, Y_train, n_epoch=NUM_EPOCH, batch_size=BATCH_SIZE)\n",
    "    \n",
    "    y_pred = clf.predict(X_test, batch_size=BATCH_SIZE)\n",
    "    print(classification_report(Y_test.ravel(), y_pred.ravel(), target_names=tag2idx.keys()))\n",
    "\n",
    "    idx2tag = {idx : tag for tag, idx in tag2idx.items()}\n",
    "    _test = [word2idx[w] for w in sample] + [0] * (SEQ_LEN-len(sample))\n",
    "    labels = clf.infer(_test, len(sample))\n",
    "    print(' '.join(sample))\n",
    "    print(' '.join([idx2tag[idx] for idx in labels if idx != 0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
